#!/usr/bin/env python
# -*- coding:utf-8 -*-

import sys, os, os.path
import time
import gzip
import base64

from jujuc.jujuc import Jujube, Server

from jujuc import log

#######################################################################

class SinkFiler(object) :
    '''
    while using cache :
        when change suffix of file ?
        when reopen an exist file ?

    solution:
        file-cell should be exist, but with file-handle closed
        when destroyed, switching all files
    '''

    MAX_LIVE_CELLS = 256

    def __init__(self, config) :
        self.config = config

        self.switchsize = config.switchsize
        self.switchline = config.switchline
        self.switchtimeout = config.switchtimeout

        self.__cells = {}
        self.__live_cells = []

        self.root_dir = '%s/file.data' % config.sinkroot
        if not os.path.exists(self.root_dir) :
            os.makedirs(self.root_dir)
        # __init__()

    def __del__(self) :
        for cid, cell in self.__cells.iteritems() :
            cell.switch()
            del cell
        # __del__()

    class FileCell(object) :
        def __init__(self, cid, root_dir, config) :
            self.switch_timeout = config.switchtimeout

            self.filer = None
            self.count_lines = 0
            self.count_bytes = 0
            self.timeout = time.time() + self.switch_timeout

            self.opened = False
            self.opened_since = 0

            self.root_dir = root_dir
            self.filepath_live = None
            self.filepath_done = None
            self.filepath_gzip = None
            self.suffix = config.suffix
            self.subdir_date = config.datesubdir
            self.gzipped = config.gzipfile

            self.cid = cid
            # __init__()

        def __del__(self) :
            self.switch()
            # __del__()

        def open(self) :
            if self.opened : return

            if self.filepath_live is None :
                if self.subdir_date :
                    self.file_dir = '%s/%05d/%s' % (self.root_dir, self.cid, time.strftime('%Y%m%d'))
                else :
                    self.file_dir = '%s/%05d' % (self.root_dir, self.cid)
                if not os.path.exists(self.file_dir) :
                    os.makedirs(self.file_dir)

                while True :
                    self.filepath_live = '%s/%05d_%s' % (self.file_dir, self.cid, time.strftime('%Y%m%d%H%M%S'))
                    self.filepath_done = ''.join( (self.filepath_live, '.', self.suffix) )
                    self.filepath_gzip = ''.join( (self.filepath_done, '.', 'gz') )
                    if os.path.exists(self.filepath_live) or os.path.exists(self.filepath_done) or os.path.exists(self.filepath_gzip) :
                        time.sleep(1)
                    else :
                        break

                self.timeout = time.time() + self.switch_timeout

            self.filer = open(self.filepath_live, 'a')
            self.opened_since = time.time()
            self.opened = True
            # open()

        def close(self) :
            if not self.opened : return

            self.filer.close()
            self.opened_since = 0
            self.opened = False
            # close()

        def switch(self) :
            #log.debug('switching FileCell #%d ...', self.cid)
            if self.opened :
                self.close()
            if self.count_bytes < 1 :
                if (self.filepath_live is not None) and os.path.exists(self.filepath_live) and (os.path.getsize(self.filepath_live) < 1) :
                    os.unlink(self.filepath_live)
                    return

            # change suffix of file
            if (self.filepath_live is not None) and os.path.exists(self.filepath_live) :
                #log.debug('renaming %s --> %s ...', self.filepath_live, self.filepath_done)
                os.rename(self.filepath_live, self.filepath_done)
                log.info('switch over %s', self.filepath_done)

            # gzip file
            if self.gzipped and (self.filepath_done is not None) and os.path.exists(self.filepath_done) :
                filer = open(self.filepath_done)
                gzip_filer = gzip.GzipFile(self.filepath_gzip, 'wb')
                gzip_filer.write( filer.read() )
                gzip_filer.close()
                filer.close()
                os.unlink(self.filepath_done)

            self.count_lines = 0
            self.count_bytes = 0

            self.filepath_live = None
            self.filepath_done = None
            self.filepath_gzip = None
            # switch()

        # class FileCell

    def switchFileCell(self, cid) :
        if cid not in self.__cells :
            raise RuntimeError, 'Invalid CID "%d" to switch.' % cid

        cell = self.__cells[ cid ]
        cell.switch()

        if cell in self.__live_cells :
            self.__live_cells.remove(cell)
        # switchFileCell()

    def openFileCell(self, cid) :
        if cid not in self.__cells :
            cell = self.FileCell(cid, self.root_dir, self.config)
            self.__cells[ cid ] = cell

        cell = self.__cells[ cid ]
        if not cell.opened :
            cell.open()
        # openFileCell()

    def closeFileCell(self, cid) :
        if cid not in self.__cells : return False

        cell = self.__cells[ cid ]
        if cell.opened :
            cell.close()

        if cell in self.__live_cells :
            self.__live_cells.remove(cell)

        return True
        # closeFileCell()

    def getFileCell(self, cid) :
        if cid not in self.__cells :
            self.openFileCell(cid)

        cell = self.__cells[ cid ]
        if not cell.opened :
            cell.open()

        if cell in self.__live_cells :
            self.__live_cells.remove(cell)
        self.__live_cells.append(cell)
        while len(self.__live_cells) > self.MAX_LIVE_CELLS :
            _cell = self.__live_cells.pop(0)
            _cell.close()

        return cell
        # getFileCell()

    def write(self, cid, data_list) :
        file_cell = self.getFileCell(cid)
        file_cell.count_lines += len(data_list)
        content = '\n'.join(data_list)
        file_cell.count_bytes += len(content) + 1

        #log.debug('writing cid:%d "%s" lines:%d bytes:%d', cid, content, file_cell.count_lines, file_cell.count_bytes )
        log.debug('writing cid:%d content-size:%d lines:%d bytes:%d', cid, len(content), file_cell.count_lines, file_cell.count_bytes )

        filer = file_cell.filer
        filer.write(content + '\n')
        filer.flush()

        if file_cell.timeout < time.time() :
            log.info('switch file cell #%d by time(timeout:%.2f now:%.2f)', cid, file_cell.timeout, time.time())
            self.switchFileCell(cid)
            return

        if (file_cell.count_lines > self.switchline) or (file_cell.count_bytes > self.switchsize) :
            log.info('switch file cell #%d by lines(lines:%d switch:%d bytes:%d switch:%d)', cid, file_cell.count_lines, self.switchline, file_cell.count_bytes, self.switchsize)
            self.switchFileCell(cid)
            return
        # write()

    def OnPeriodic(self) :
        log.debug('%s::OnPeriodic()', self.__class__.__name__)
        now = time.time()
        for cid, cell in self.__cells.iteritems() :
            if cell.timeout < now :
                #log.debug('switch file cell #%d by time(timeout:%.2f now:%.2f)', cid, cell.timeout, now)
                cell.switch()
        # OnPeriodic()
    # class SinkFiler

class SinkCache(object) :
    def __init__(self, sinker, capacity=1048576) :
        self.sinker = sinker
        self.capacity = capacity
        self.cache = {}
        self.size = 0
        # __init__()

    def __del__(self) :
        self.flush()
        # __del__()

    def write(self, cid, data) :
        self.size += len(data)
        if cid in self.cache :
            self.cache[ cid ].append( data )
        else :
            self.cache[ cid ] = [ data, ]

        if self.size >= self.capacity :
            self.flush()
        # write()

    def flush(self) :
        for cid, data in self.cache.iteritems() :
            self.sinker.write(cid, data)

        self.cache = {}
        self.size = 0
        # flush()

    def OnPeriodic(self) :
        log.debug('%s::OnPeriodic()', self.__class__.__name__)
        self.flush()

        self.sinker.OnPeriodic()
        # OnPeriodic()
    # class SinkCache

class SinkServer(Server)  :
    '''Sink Consumer'''
    def __init__(self, config) :
        self.config = config

        self.sinker = SinkFiler(config)
        self.cache = SinkCache(self.sinker, config.outcachesize)
        # __init__()

    def __del__(self) :
        try :
            del self.cache
        except :
            log.error('del CACHE: %s', sys.exc_info())

        try :
            del self.sinker
        except :
            log.error('del SINKER: %s', sys.exc_info())

        # __del__()

    def serve(self, service, record, address_from) :
        '''
        INCOME REORD FORMAT: <cid>\t<data>
        '''
        record = base64.b64decode(record)
        cid, data = record.split('\t', 1)
        cid = int(cid)
        if data :
            self.cache.write(cid, data)
        return None, None, None     # service_to,response,address_to
        # serve()

    def OnPeriodic(self) :
        log.debug('%s::OnPeriodic()', self.__class__.__name__)
        self.cache.OnPeriodic()
        # OnPeriodic()

    # class EchoServer


#######################################################################

class   SinkConfigure( object ) :
    def __init__(self) :
        self.port = 4344
        self.service = 'sink'
        self.sinkroot = '/tmp/sinkroot'

        self.switchsize    = 1073741824
        self.switchline    = 1048576
        self.switchtimeout = 120
        self.outcachesize  = 4194304
        self.suffix = 'txt'

        # path pattern :
        # /path/to/sinkroot/file.data/<cid>/<yyyymmdd>/<cid>_<yyyymmddHHMMSS>.<suffix>
        self.cidprefix  = False
        self.datesubdir = False
        self.gzipfile   = False
        # __init__()

    def parse(self, configure) :
        from ConfigParser import ConfigParser

        config = ConfigParser()
        config.readfp(open(configure))

        SECTION = 'sink.file'
        option = 'sink.file.port'
        if config.has_option(SECTION, option) :
            self.port = config.getint(SECTION, option)
        option = 'sink.file.service'
        if config.has_option(SECTION, option) :
            self.service = config.get(SECTION, option)
        option = 'sink.file.sinkroot'
        if config.has_option(SECTION, option) :
            self.sinkroot = config.get(SECTION, option)

        option = 'sink.file.switchsize'
        if config.has_option(SECTION, option) :
            self.switchsize = config.getint(SECTION, option)
        option = 'sink.file.switchline'
        if config.has_option(SECTION, option) :
            self.switchline = config.getint(SECTION, option)
        option = 'sink.file.switchtimeout'
        if config.has_option(SECTION, option) :
            self.switchtimeout = config.getint(SECTION, option)
        option = 'sink.file.outcachesize'
        if config.has_option(SECTION, option) :
            self.outcachesize = config.getint(SECTION, option)
        option = 'sink.file.suffix'
        if config.has_option(SECTION, option) :
            self.suffix = config.get(SECTION, option)

        option = 'sink.file.cidprefix'
        if config.has_option(SECTION, option) :
            cidprefix = config.get(SECTION, option).lower()
            if 'y'==cidprefix or 'yes'==cidprefix or 't'==cidprefix or 'true'==cidprefix :
                self.cidprefix = True
            else :
                self.cidprefix = False
        option = 'sink.file.datesubdir'
        if config.has_option(SECTION, option) :
            datesubdir = config.get(SECTION, option).lower()
            if 'y'==datesubdir or 'yes'==datesubdir or 't'==datesubdir or 'true'==datesubdir :
                self.datesubdir = True
            else :
                self.datesubdir = False
        option = 'sink.file.gzipfile'
        if config.has_option(SECTION, option) :
            gzipfile = config.get(SECTION, option).lower()
            if 'y'==gzipfile or 'yes'==gzipfile or 't'==gzipfile or 'true'==gzipfile :
                self.gzipfile = True
            else :
                self.gzipfile = False
        # parse()

    def __str__(self) :
        return '<SinkConfigure port:%d service:%s sinkroot:%s switch(size:%d line:%d timeout:%d cache-size:%d suffix:%s) cidprefix:%s date-subdir:%s gzipped:%s' % (
                self.port, self.service, self.sinkroot,
                self.switchsize, self.switchline, self.switchtimeout, self.outcachesize, self.suffix,
                self.cidprefix, self.datesubdir, self.gzipfile )
        # __str__
    # class SinkConfigure

def main():
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
                        help='show detail information')
    parser.add_option("", "--conf", dest="config", type="string",
                        help='configure file')
    parser.add_option("", "--log-dir", dest="log_dir", type="string",
                        help='where cache files locate')
    (options, args) = parser.parse_args()

    if options.log_dir :
        log.setLogPath( options.log_dir )
    if options.verbose :
        log.enableDebug()
    else :
        log.disableDebug()

    config = SinkConfigure()
    if options.config :
        config.parse(options.config)

    s = Jujube()
    s.listenTCP(config.port)

    server = SinkServer(config)
    s.registerService(config.service, server)
    s.registerPeriodic(server, 60.0)
    s.run()
    s.stop()
    # main()

if __name__ == '__main__' :
    main()

    import sys
    sys.exit(0)

